#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: xaoyaoyao
@contact: xaoyaoyao@aliyun.com
@file: tmtpost_spider.py
@time: 2018/08/25
"""
import random

from scrapy.http import Request
from scrapy.linkextractors import LinkExtractor
from scrapy.selector import Selector

from article_spider.items import ArticleItem
from article_spider.spiders.dynamic.js_spider import JSSpider


class TmtpostSpider(JSSpider):
    name = "tmtpost"
    allowed_domains = ["tmtpost.com"]
    start_urls = [
        'https://www.tmtpost.com/',
        'http://www.tmtpost.com/column/3189960',
        'http://www.tmtpost.com/column/2573550',
        'http://www.tmtpost.com/column/2446153',
        'http://www.tmtpost.com/column/3015019',
        'http://www.tmtpost.com/column/2446155',
        'http://www.tmtpost.com/column/2446157',
        'http://www.tmtpost.com/column/2581216',
        'http://www.tmtpost.com/column/2573543',
    ]

    def __init__(self):
        super(TmtpostSpider, self).__init__()

    def parse(self, response):
        response_selector = Selector(response=response)
        current_url = response.url
        self.logger.info('This is an item page! %s', current_url)
        nav_title = response_selector.xpath(
                u'//div[@class="inner"]//ul[contains(@class, "column-list")]/li/a[@class="current"]/text()').extract()
        if nav_title is None or len(nav_title) <= 0:
            nav_title = response_selector.xpath(
                    u'//div[@class="nav_cont"]//h2[@class="nav_title"]//a[contains(@class, "current")]/text()').extract()
        _type = ''
        if nav_title and len(nav_title) > 0:
            _type = nav_title[0]
        try:
            d_link = LinkExtractor(
                    restrict_xpaths='//div[@class="user-article-list"]/div[contains(@class, "mod-article-list")]/ul/li/div[@class="cont"]/h3/a')
            if d_link:
                links = d_link.extract_links(response)
                if links is None or len(links) <= 0:
                    dd_link = LinkExtractor(
                            restrict_xpaths='//ul[@class="article-list"]/li[contains(@class, "post_part")]/div[@class="cont"]/h3/a')
                    if dd_link:
                        links = dd_link.extract_links(response)
                if links:
                    for detail_link in links:
                        if detail_link:
                            url = detail_link.url
                            if url and '/photoset/' in url:
                                continue
                            yield Request(url=url, callback=self.parse_detail, meta={'_type': _type})
        except Exception as e:
            self.logger.error("the link detail error. The msg %s", str(e))

    def parse_detail(self, response):
        article_item = ArticleItem()
        response_selector = Selector(response=response)
        current_url = response.url
        self.logger.info('This is an item page! %s', current_url)
        article_item['url'] = current_url
        # title
        rt_title = response_selector.xpath(
                u'//div[@class="container"]//div[contains(@class, "post-container")]/article/h1/text()').extract()
        title = ''
        if rt_title and len(rt_title) > 0:
            title = rt_title[0]
        article_item['title'] = title
        # content
        rt_content = response_selector.xpath(
                u'//div[@class="container"]//div[contains(@class, "post-container")]//div[@class="inner"]/p/text()').extract()
        content = ''
        if rt_content:
            for c_text in rt_content:
                content += str(c_text) + '\n'
                content = content.replace('　', '')
        article_item['content'] = content
        ## author
        rt_author = response.css('div.container div.post-container.c-page div.post-info a.color-orange::text').extract()
        author = ''
        if rt_author and len(rt_author) > 0:
            author = rt_author[0]
        article_item['author'] = author
        rt_date = response.css('div.container div.post-container.c-page div.post-info span.time::text').extract()
        _date = None
        if rt_date:
            if rt_date and len(rt_date) > 0:
                _date = rt_date[0]
        article_item['type'] = response.meta['_type']
        article_item['name'] = '钛媒体 '
        article_item['date'] = _date
        article_item['grade'] = random.randint(88, 95)
        article_item['domain'] = 'tmtpost.com'
        self.logger.info('article_item >> %s', article_item)
        yield article_item
